package org.zjt.spark.logs

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

import scala.util.matching.Regex

/**
  * DESC    本地统计异常（正则）
  *
  **/
object ExceptionLogb extends App {
  val LOCAL_FILE_PREFIX = "file://"
  val HDFS_FILE_PREFIX = "hdfs:/"


  // localLogHandler()

  def localLogHandler() {
    val conf = new SparkConf().setMaster("local[2]").setAppName("NetworkWordCount").set("spark.driver.allowMultipleContexts", "true")
    val sc = new SparkContext(conf)
    val pattern = new Regex("\\w+(E|e)xception:")
    val lines: RDD[String] = sc.textFile(LOCAL_FILE_PREFIX + "/opt/logs/100003171/apollo-configservice.log.2")
    lines.coalesce(2)
    lines.persist()
    println(lines.collect().mkString("\n"))
    val result: RDD[(String, Int)] = lines.filter(_.contains("Caused by:")).flatMap((pattern findFirstIn _)).map((_, 1)).reduceByKey(_ + _)
    println(result.collect().mkString("\n"))
    sc.stop()
  }


  var hdfsLogHandler = () => {
    println("-----------hdfsLogHandler------------")
    val conf = new SparkConf().setMaster("local[2]").setAppName("NetworkWordCount").set("spark.driver.allowMultipleContexts", "true")
    val sc = new SparkContext(conf)
    val pattern = new Regex("\\w+(E|e)xception:")
    val lines: RDD[String] = sc.textFile(HDFS_FILE_PREFIX + "/centos:9000/user/apollo-configservice.log")
    lines.coalesce(7)
    println(lines.partitions.length)
    lines.persist()
    println(lines.collect().mkString("\n"))
    val result: RDD[(String, Int)] = lines.filter(_.contains("Caused by:")).flatMap((pattern findFirstIn _)).map((_, 1)).reduceByKey(_ + _)
    println(result.collect().mkString("\n"))
    sc.stop()
  }
  hdfsLogHandler()


}


object Test {
  def main(args: Array[String]) {

    val pattern = new Regex("\\w+(E|e)xception:") // 首字母可以是大写 S 或小写 s
    val str = "Caused by: com.mysql.jdbc.exceptions.jdbc4.CommunicationsException: Communications link failure"
    println((pattern findAllIn str).mkString(",")) // 使用逗号 , 连接返回结果
    val z:Array[String] = new Array[String](3)
    for( i <- 1 until 3){
      z(i) = "zhang"+i
    }
    z.drop(0)
    println(z.mkString(","))
    println(z.take(2).mkString(","))

  }
}